#!/usr/bin/python

path="bnc/"
lists=[]

for i in range(1,15):
	fn = path+("basewrd%d.txt" % i)
	f = open(fn, "rt")

	for line in f:
		line = line.replace("\r", "").replace("\n", "")
		if len(line) == 0:
			continue
		if line[0] != '\t':
			lists.append([i, line.split()[0]])
		else:
			lists[-1].append(line.split()[0])

for i in range(15, 21):
	fn = path+("basewrd%d.txt" % i)
	f = open(fn, "rt")

	for line in f:
		line = line.replace("\r", "").replace("\n", "")
		line = line.replace("> ", "")
		if len(line) == 0:
			continue
		lists.append([i] + line.split())
		
# make words lowercase
lists = map(lambda x: [x[0]] + map(lambda y: y.lower(), x[1:]), lists)

word_family = []
word_full = []

for i in range(len(lists)):
	w = lists[i]
	word_family.append([w[0], w[1]])
	word_full += zip(w[1:], [i for k in range(len(w)-1)])

word_full.sort(key=lambda x: x[0])
print word_family

